{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LightGBM Parameter Tuning for Otto Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 直接调用LightGBM内嵌的cv寻找最佳的参数n_estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，进行XGBoost参数调优探索。\n",
    "\n",
    "竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from xgboost import XGBClassifier\n",
    "import lightgbm as lgbm\n",
    "#from lgbm.sklearn import LGBMClassifier\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"Otto_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_84</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.00000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>30939.500000</td>\n",
       "      <td>0.38668</td>\n",
       "      <td>0.263066</td>\n",
       "      <td>0.901467</td>\n",
       "      <td>0.779081</td>\n",
       "      <td>0.071043</td>\n",
       "      <td>0.025696</td>\n",
       "      <td>0.193704</td>\n",
       "      <td>0.662433</td>\n",
       "      <td>1.011296</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070752</td>\n",
       "      <td>0.532306</td>\n",
       "      <td>1.128576</td>\n",
       "      <td>0.393549</td>\n",
       "      <td>0.874915</td>\n",
       "      <td>0.457772</td>\n",
       "      <td>0.812421</td>\n",
       "      <td>0.264941</td>\n",
       "      <td>0.380119</td>\n",
       "      <td>0.126135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>17862.784315</td>\n",
       "      <td>1.52533</td>\n",
       "      <td>1.252073</td>\n",
       "      <td>2.934818</td>\n",
       "      <td>2.788005</td>\n",
       "      <td>0.438902</td>\n",
       "      <td>0.215333</td>\n",
       "      <td>1.030102</td>\n",
       "      <td>2.255770</td>\n",
       "      <td>3.474822</td>\n",
       "      <td>...</td>\n",
       "      <td>1.151460</td>\n",
       "      <td>1.900438</td>\n",
       "      <td>2.681554</td>\n",
       "      <td>1.575455</td>\n",
       "      <td>2.115466</td>\n",
       "      <td>1.527385</td>\n",
       "      <td>4.597804</td>\n",
       "      <td>2.045646</td>\n",
       "      <td>0.982385</td>\n",
       "      <td>1.201720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>15470.250000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>30939.500000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>46408.750000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61.00000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>130.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>87.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 94 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id       feat_1        feat_2        feat_3        feat_4  \\\n",
       "count  61878.000000  61878.00000  61878.000000  61878.000000  61878.000000   \n",
       "mean   30939.500000      0.38668      0.263066      0.901467      0.779081   \n",
       "std    17862.784315      1.52533      1.252073      2.934818      2.788005   \n",
       "min        1.000000      0.00000      0.000000      0.000000      0.000000   \n",
       "25%    15470.250000      0.00000      0.000000      0.000000      0.000000   \n",
       "50%    30939.500000      0.00000      0.000000      0.000000      0.000000   \n",
       "75%    46408.750000      0.00000      0.000000      0.000000      0.000000   \n",
       "max    61878.000000     61.00000     51.000000     64.000000     70.000000   \n",
       "\n",
       "             feat_5        feat_6        feat_7        feat_8        feat_9  \\\n",
       "count  61878.000000  61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       0.071043      0.025696      0.193704      0.662433      1.011296   \n",
       "std        0.438902      0.215333      1.030102      2.255770      3.474822   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      1.000000      0.000000   \n",
       "max       19.000000     10.000000     38.000000     76.000000     43.000000   \n",
       "\n",
       "           ...            feat_84       feat_85       feat_86       feat_87  \\\n",
       "count      ...       61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       ...           0.070752      0.532306      1.128576      0.393549   \n",
       "std        ...           1.151460      1.900438      2.681554      1.575455   \n",
       "min        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "25%        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "50%        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "75%        ...           0.000000      0.000000      1.000000      0.000000   \n",
       "max        ...          76.000000     55.000000     65.000000     67.000000   \n",
       "\n",
       "            feat_88       feat_89       feat_90       feat_91       feat_92  \\\n",
       "count  61878.000000  61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       0.874915      0.457772      0.812421      0.264941      0.380119   \n",
       "std        2.115466      1.527385      4.597804      2.045646      0.982385   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        1.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max       30.000000     61.000000    130.000000     52.000000     19.000000   \n",
       "\n",
       "            feat_93  \n",
       "count  61878.000000  \n",
       "mean       0.126135  \n",
       "std        1.201720  \n",
       "min        0.000000  \n",
       "25%        0.000000  \n",
       "50%        0.000000  \n",
       "75%        0.000000  \n",
       "max       87.000000  \n",
       "\n",
       "[8 rows x 94 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集是因为的数据特征单一，我们可以在特征工程方面少做些工作，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['target']\n",
    "y_train = y_train.map(lambda s: s[6:])\n",
    "y_train = y_train.map(lambda s: int(s)-1)\n",
    "\n",
    "train = train.drop([\"id\", \"target\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围\n",
    "（采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#直接调用LightGBM内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "import json\n",
    "def modelfit(params, alg, X_train, y_train, early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "    lgbm_params['num_class'] = 9\n",
    "    \n",
    "    #直接调用LightGBM，而非sklarn的wrapper类\n",
    "    lgbmtrain = lgbm.Dataset(X_train, y_train, silent=True)\n",
    "    \n",
    "    cv_result = lgbm.cv(\n",
    "        lgbm_params, lgbmtrain, num_boost_round=10000, nfold=5, stratified=False, shuffle=True, metrics='multi_logloss',\n",
    "        early_stopping_rounds=early_stopping_rounds,show_stdv=True,seed=0)\n",
    "    # note: cv_results will look like: {\"multi_logloss-mean\": <a list of historical mean>,\n",
    "    # \"multi_logloss-stdv\": <a list of historical standard deviation>}\n",
    "    print('best n_estimators:', len(cv_result['multi_logloss-mean']))\n",
    "    print('best cv score:', cv_result['multi_logloss-mean'][-1])\n",
    "    #cv_result.to_csv('lgbm1_nestimators.csv', index_label = 'n_estimators')\n",
    "    json.dump(cv_result, open('lgbm_1.json', 'w'))\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = len(cv_result['multi_logloss-mean']))\n",
    "    alg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('best n_estimators:', 539)\n",
      "('best cv score:', 0.57499165152258824)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/lightgbm/sklearn.py:282: LGBMDeprecationWarning: The `seed` parameter is deprecated and will be removed in next version. Please use `random_state` instead.\n",
      "  'Please use `random_state` instead.', LGBMDeprecationWarning)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/lightgbm/sklearn.py:285: LGBMDeprecationWarning: The `nthread` parameter is deprecated and will be removed in next version. Please use `n_jobs` instead.\n",
      "  'Please use `n_jobs` instead.', LGBMDeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'multiclass', \n",
    "          'nthread': -1, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 6,\n",
    "          'max_bin': 127, \n",
    "          'subsample_for_bin': 50000,\n",
    "          'subsample': 0.8, \n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 1, \n",
    "          'reg_lambda': 0,\n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1, \n",
    "          'min_child_samples': 20, \n",
    "          'scale_pos_weight': 1}\n",
    "\n",
    "lgbm1 = lgbm.sklearn.LGBMClassifier(num_class= 9, n_estimators=1000, seed=0, **params)\n",
    "\n",
    "modelfit(params,lgbm1, X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcHHWd//HXu6fnTibnJOQkQFCI\nB1GjAq4reCCwCq4HkuVQV2XZxVt/Kv52PdjDG9GfJyrkoSIoigqsN6KoIBLuhAiEACEkJJM7mWQy\n1+f3R1V3OpOemZ5kejoz/X4+0o/uqvpW1ad6Ov3p7/dbVV9FBGZmZgCZSgdgZmaHDicFMzPLc1Iw\nM7M8JwUzM8tzUjAzszwnBTMzy3NSGOUkvVjSgyWWPUnSmnLHNFZImitpp6SaSsdiNlKcFEYJSY9J\nennf+RHxx4h4+jDtY4mk/yoy/2xJt0tql7Qhff1vklSwXmf6BbpD0p2SXlKw/pslhaRL+2z3Nen8\nJcMR/8Hq+x5HxOqIGBcRPWXY18clfW+4t1tO/X0Gh3kfo+59GWucFGxAkt4PfBH4LHAYMB24EHgR\nUFdQ9DMRMQ6YAHwNuK7PL+xHgDdKyhbMOx94qIzhj1l93kezYeOkMMr1bRKS9FxJd6e/2K+V9IO+\nv/4lvT/9xb9O0lvSeRcA5wAfTH/x3yBpAnAJ8G8R8aOI2BGJuyPinIjY0zeeiOgFvg9MJkkgOU8B\n9wOvTPc3GTgRuH6wYysW7yDvSb2kz0laLWm9pK9LakyXTZV0o6StkjZL+qOkjKTvAnOBG9Lj/6Ck\neWlNJpuu+3tJ/yXp1oL3aIqkqyRtl3SHpHkFcXxR0hPpsjslvTidfyrwEZIkuVPSven8mZKuT+Na\nKentBdv6uKQfSfqepO3AmyW9QNLSdPvr+9bECtZdIelVBdNZSRvTz0pDus1N6Xtyh6TpxbYzyHv+\n9jTmzekxzCxYdoqkByVtk/RVSX+Q9LYD2Mex6d9gq6Tlks4oWHa6pAfSz/2Tkj6Qzi/69x7qvquJ\n35wxRFId8BNgCcmX8tXAP/YpdhjJr/lZwFuBr0iaFBGXA1eR/uKPiFcDJwD1wM+GEEMNSQ3gUWB9\nn8XfSZcBnJ1ud7/EUkq8g6zzaeBpwEJgfrruR9Nl7wfWAK0kSesjQETEecBq4NXp8X+mn22fDZyX\nbvMo4DbgSpL3ewXwsYKyd6QxTCZJlNdKaoiIXwL/A/wg3ddxafmr09hmAq8H/kfSywq2dybwI2Ai\nyd/qi8AXI6IljeWH/cR8NbC4YPqVwMaIuAt4E8n7OweYQlIL3N3PdoqS9FLgk8BZwAzgceCadNnU\nNOaL0+0/SPJjYEgk1QI3AL8GpgHvBK6SlGs6/TbwLxExHngm8Lt0ftG/91D3X02cFMaW44Es8KWI\n6IqI64C/9inTBVySLv85sBPor09iKsmXR3duRvoreauk3ZL+vqDsByRtBdqBy4D/KNIW/xPgpLQG\ncj5JkhjMUOJFkoC3A++NiM0RsYPkC/jsgu3NAA5Pt/nHGNoNwK6MiEciYhvwC+CRiPht+h5dCzwn\nVzAivhcRmyKiOyI+T5Jgi8YuaQ7wd8CHIqIjIu4BvkWSgHJui4ifRkRvROxOj2W+pKkRsTMi/tJP\nzN8HzpDUlE7/Uzov935MAeZHRE9E3BkR24fwfkBSw7wiIu5Ka48XAyektabTgeURcV36Hn2JpNY4\nVMcD44BPRURnRPwOuJG9ya4LWCCpJSK2pAkvN/9g/t5Vx0lhbJkJPNnnQ/9EnzKbCr/kgV0k/9mK\n2QRMVUH7dUScGBET02WFn5/PpfMbgUXAZyWdVrix9Ivsf4F/B6ZGxJ9LOKahxAvJL8Im4M40eW0F\nfpnOh6RvZCXwa0mrJH24hBgKFdZ+dheZzseWNnutSJtNtpL8Ip/az3ZnArkklvM4SY0kp+/f8q0k\nNaK/pc0+r6KIiFhJUot5dZoYzmBvUvgu8CvgGklrJX0m/VU+FDPTWHP720ny+ZiVLnuiYFmQ/HIf\nqpnAE2nzZE7h+/M6kgT0eNo8dUI6/2D/3lXHSWFsWQfMSn8t58wZwvp9f0HdRtK8c2bJG0gsA/4M\n/EORIt8hqdJ/dwhxDcVGki/nZ0TExPQxIe0EJ+0XeX9EHAm8GnhfQRPNsP2CTPsPPkTSpDIpTZjb\ngNzfpu++1gKTJY0vmDcXeLJgep91IuLhiFhM0pzyaeBHkpr7CSnXhHQm8ECaKEh/PX8iIhaQNOu8\nir1NfKVaCxyem0hjmJLGvg6YXbBMhdND3MecPv0B+fcnIu6IiDNJ3oufkjalDfL3tiKcFEaX2rRj\nMPfoewbKbUAP8I60M/FM4AVD2P564MjcRERsBT4BfFXS6yWNU9IpuxDo78sHSceQNIUsL7L4D8Ar\ngP83hLhKlv6S/CbwBUnT0nhmScp1cL9K0vz0y2k7yfuVa+ba5/gP0nigG2gDspI+CrQULF8PzMt9\nyUXEE8CtwCfTv+2zSWoCV/W3A0nnSmpNj3lrOru/02evAU4B/pW9tQQknSzpWWlf0HaS5paBTsEt\n9hn8PvAWSQsl1ZM0190eEY+R1AyfpeT04yxwEUk/0UAyffZRD9xO0jT5QUm1kk4i+ZK/RlKdpHMk\nTYiILvb+XQf7e1sRTgqjy89JfgXnHh8vXBgRncBrSb5MtgLnkrS7DtaZm/NtknbZrZJ+mm7zM8D7\ngA8CG0i+zL5B8iv41oJ1c2cttZN0Bl6ZlttHWpO4KSI2lxjTgfgQSZPBX5ScqfNb9rblH51O7yRJ\nol+NiN+nyz4J/Ht6/B84yBh+RdLn8BBJM0cH+zb/XJs+b5KUa/9eDMwj+VX8E+BjEfGbAfZxKrBc\n0k6STuezI6KjWMGIWEdyvCcCPyhYdBhJR/B2kiamPwADXSew32cwIm4C/gP4MUnN4CjSPpyI2Ai8\nAfgMSZPSAmApA38mF/fZxyPpZ/sM4DSS2uBXgfMj4m/pOucBj6V/7wtJPvsw8N/bipD7XMY2SbcD\nX4+IKysdi1laM1oDnBMRN1c6HtufawpjjKSXSDosbT56E/Bsko5Ws4qQ9EpJE9NmoI+Q9Kv0d6aU\nVZiTwtjzdOBekk7N9wOvT5sOxhQlFy/tLPI4p9Kx2X5OILmifSNJP8Br0jPR7BDk5iMzM8tzTcHM\nzPJG3U21pk6dGvPmzat0GGZmo8qdd965MSJaBys36pLCvHnzWLp0aaXDMDMbVSQ9PngpNx+ZmVkB\nJwUzM8tzUjAzszwnBTMzy3NSMDOzPCcFMzPLK1tSkDRH0s3pICPLJb27SBlJ+pKSsV3vk/TccsVj\nZmaDK2dNoRt4f0QcSzKU3kWSFvQpcxrJrW2PBi4AvlauYB7b2M53bnuMbbu6yrULM7NRr2xJISLW\n5cZJTYcYXMG+QwtCMgrUd9J77P8FmChpRjnieWDddj76s+U8tb3o7ebNzIwR6lNIB/B+DsnoSYVm\nse/AI2vYP3Eg6QJJSyUtbWtrO6AY6rPJoXZ0edAlM7P+lD0pSBpHMiLTeyJie9/FRVbZ77atEXF5\nRCyKiEWtrYPeuqOo+mwNAHu6ewcpaWZWvcqaFCTVkiSEqyLiuiJF1rDvwPKzSYYiHHb1tcmh7ul2\nTcHMrD/lPPtIJGP+roiIS/spdj1wfnoW0vHAtnINCNOQqyl0uaZgZtafct4l9UUkg2nfL+medN5H\ngLkAEfF1kkHATycZZH0X8JZyBZOrKXS4pmBm1q+yJYWI+BPF+wwKywRwUbliKJTraHZNwcysf1Vz\nRbM7ms3MBlc1SaHBHc1mZoOqmqSQqyl0uPnIzKxfVZQUXFMwMxtM1SSFTEbU1WTcp2BmNoCqSQqQ\n1BZ89pGZWf+qKynUZnydgpnZAKorKWRrXFMwMxtAdSWF2ow7ms3MBlBdSSFb445mM7MBVFlSyHg8\nBTOzAVRdUnBNwcysf1WVFBpq3XxkZjaQqkoKyXUKbj4yM+tPdSUF1xTMzAZUXUnBNQUzswFVVVJo\nqHVHs5nZQKoqKdRna3xKqpnZAKosKbimYGY2kLIlBUlXSNogaVk/yydIukHSvZKWS3pLuWLJqc/W\n0N0bdPc4MZiZFVPOmsIS4NQBll8EPBARxwEnAZ+XVFfGePJDcnY6KZiZFVW2pBARtwCbByoCjJck\nYFxatrtc8cDe0dc8JKeZWXGV7FP4MnAssBa4H3h3RBT9tpZ0gaSlkpa2tbUd8A7ra5Nxmn2nVDOz\n4iqZFF4J3APMBBYCX5bUUqxgRFweEYsiYlFra+sB7zA/TrNrCmZmRVUyKbwFuC4SK4FHgWPKucOG\nfE3BScHMrJhKJoXVwMsAJE0Hng6sKucO9/YpuPnIzKyYbLk2LOlqkrOKpkpaA3wMqAWIiK8D/wks\nkXQ/IOBDEbGxXPFAckoquKZgZtafsiWFiFg8yPK1wCnl2n8x9ekpqe5oNjMrrqquaG5Iawo+JdXM\nrLiqSgqNdUlS2NVZ1sshzMxGrapKCk1pUtjd6eYjM7NiqjIptDspmJkVVWVJIelX3+3mIzOzoqoq\nKdRlM2QzYpdrCmZmRVVVUoCks9lJwcysuKpLCs11WZ99ZGbWj6pLCk2uKZiZ9avqkkJjXY1PSTUz\n60fVJYXmuiztbj4yMyuq6pKCawpmZv2ruqTgPgUzs/5VXVLwKalmZv2ruqTgU1LNzPpXdUnBzUdm\nZv2ruqTQWFfDnu5eenqj0qGYmR1yqi4pNKc3xXMTkpnZ/sqWFCRdIWmDpGUDlDlJ0j2Slkv6Q7li\nKdToMRXMzPpVzprCEuDU/hZKmgh8FTgjIp4BvKGMseQ15Udfc1IwM+urbEkhIm4BNg9Q5J+A6yJi\ndVp+Q7liKdSUbz5yUjAz66uSfQpPAyZJ+r2kOyWd319BSRdIWippaVtb20HttMnjNJuZ9auSSSEL\nPA/4B+CVwH9IelqxghFxeUQsiohFra2tB7VTNx+ZmfUvW8F9rwE2RkQ70C7pFuA44KFy7rTRScHM\nrF+VrCn8DHixpKykJuCFwIpy7zR3SuruLjcfmZn1VbaagqSrgZOAqZLWAB8DagEi4usRsULSL4H7\ngF7gWxHR7+mrwyXXfNS+xzUFM7O+ypYUImJxCWU+C3y2XDEUM64hOeSde1xTMDPrq+quaG6srSGb\nETs6uiodipnZIafqkoIkWhpr2bbbScHMrK+qSwoALQ1Ztu9285GZWV9VmRQmuKZgZlZUVSaFlsZa\ntrtPwcxsP9WZFBpq2e6agpnZfqozKTRm2eY+BTOz/VRpUnDzkZlZMdWZFBpq6ezupaPLVzWbmRWq\nzqTQWAvgfgUzsz6qMilMyCUFNyGZme2jKpNCS3r/I3c2m5ntqzqTgpuPzMyKqs6k0ODmIzOzYqoy\nKUxwTcHMrKiqTArj830KTgpmZoWqMik01NZQn804KZiZ9TFoUpB0lKT69PVJkt4laWL5QyuvSU11\nbN3lpGBmVqiUmsKPgR5J84FvA0cA3y9rVCNgUnMdW3Z1VjoMM7NDSilJoTciuoF/BC6LiPcCMwZb\nSdIVkjZIWjZIuedL6pH0+tJCHh6TmmrZ3O6kYGZWqJSk0CVpMfAm4MZ0Xm0J6y0BTh2ogKQa4NPA\nr0rY3rCa1OzmIzOzvkpJCm8BTgD+OyIelXQE8L3BVoqIW4DNgxR7J0nz1IYS4hhWk5vq2OzmIzOz\nfWQHKxARDwDvApA0CRgfEZ862B1LmkXSJPVS4PmDlL0AuABg7ty5B7trIKkpbNvdRXdPL9maqjwJ\ny8xsP6WcffR7SS2SJgP3AldKunQY9n0Z8KGIGPT+1RFxeUQsiohFra2tw7BrmNxUS4SvVTAzKzRo\nTQGYEBHbJb0NuDIiPibpvmHY9yLgGkkAU4HTJXVHxE+HYduDmtRcB8CWXZ1MGVc/Ers0MzvklZIU\nspJmAGcB/3e4dhwRR+ReS1oC3DhSCQFgcj4puKZgZpZTSlK4hOTsoD9HxB2SjgQeHmwlSVcDJwFT\nJa0BPkZ61lJEfP2AIx4mk5qSpODTUs3M9iqlo/la4NqC6VXA60pYb3GpQUTEm0stO1zyzUdOCmZm\neaV0NM+W9JP0QrT1kn4safZIBFdOk3M1BZ+WamaWV8q5mFcC1wMzgVnADem8Ua2xroaG2oxrCmZm\nBUpJCq0RcWVEdKePJcDwnBdaYZOb6tjc7o5mM7OcUpLCRknnSqpJH+cCm8od2EiY1FzH5vY9lQ7D\nzOyQUUpS+GeS01GfAtYBrye59cWoN218PRt2OCmYmeUMmhQiYnVEnBERrRExLSJeA7x2BGIru+kt\nDazf7qRgZpZzoDf9ed+wRlEh01oa2NS+h+6e3kqHYmZ2SDjQpKBhjaJCprfUEwEbd/oMJDMzOPCk\nEMMaRYVMG98AwPrtHRWOxMzs0NDvFc2SdlD8y19AY9kiGkHTW5Ib4TkpmJkl+k0KETF+JAOphOkt\naU3BZyCZmQEH3nw0JkxpriMjaHNNwcwMqPKkkK3JMGVcvU9LNTNLVXVSgKRfYf0O1xTMzMBJgenj\nfQGbmVlOKbfO3iFpe5/HE+nttI8ciSDLaVpLA22uKZiZAaWNvHYpsBb4PsnpqGcDhwEPAleQjK42\nak0bX8/GnZ109fRSW1P1FSczq3KlfAueGhHfiIgdEbE9Ii4HTo+IHwCTyhxf2eVOS23zaalmZiUl\nhV5JZ0nKpI+zCpb1e2WzpCvS0dqW9bP8HEn3pY9bJR031OCHgy9gMzPbq5SkcA5wHrAhfZwHnCup\nEXjHAOstAU4dYPmjwEsi4tnAfwKXlxLwcMvVFHwLbTOzEvoUImIV8Op+Fv9pgPVukTRvgOW3Fkz+\nBajIuM+5pLB26+5K7N7M7JBSytlHs9MzjTZIWi/px5KG+wv8rcAvhnmbJZk6ro6muhpWb95Vid2b\nmR1SSmk+uhK4HpgJzAJuSOcNC0knkySFDw1Q5gJJSyUtbWtrG65d57bN3MlNrN7kpGBmVkpSaI2I\nKyOiO30sAVqHY+eSng18CzgzIvod9zkiLo+IRRGxqLV1WHa9j7mTm1xTMDOjtKSwUdK5kmrSx7lA\nv1/gpZI0F7gOOC8iHjrY7R2MXFLo7R0Tw0SYmR2wUi5e+2fgy8AXSE5BvRV4y2ArSbqa5MK2qZLW\nAB8DagEi4uvAR4EpwFclAXRHxKKhH8LBO3xKE3u6e9mwYw+HTWioRAhmZoeEUs4+Wg2cUThP0nuA\nywZZb/Egy98GvK2EGMtu7pRmAB7f1O6kYGZV7UDv6/C+YY2iwg6f3ATgfgUzq3oHmhQ0rFFU2KxJ\njWTkpGBmdqBJYUz1yNbWZJg5sZHHfVqqmVW5fvsUJO2g+Je/gMayRVQhh09p4nHXFMysyvWbFCJi\n/EgGUmlzJzfzq+VPVToMM7OK8gACqcOnNLG5vZMdHV2VDsXMrGKcFFJz0zOQ3K9gZtXMSSE116el\nmpk5KeQcPsVJwczMSSE1vqGWyc11bj4ys6rmpFBgfus4Hl6/o9JhmJlVjJNCgQUzW1ixbrvvlmpm\nVctJocCCGS20d/b4IjYzq1pOCgUWzGwB4IG12ysciZlZZTgpFJg/bRzZjHhg3bZKh2JmVhFOCgUa\namuYP20cy11TMLMq5aTQxzNnTeD+NduIcGezmVUfJ4U+Fs6ZyKb2TtZs2V3pUMzMRpyTQh8L50wE\n4O4ntlY4EjOzkVe2pCDpCkkbJC3rZ7kkfUnSSkn3SXpuuWIZimMOG09DbYZ7VjspmFn1KWdNYQlw\n6gDLTwOOTh8XAF8rYywly9ZkeNasCdzzxJZKh2JmNuLKlhQi4hZg8wBFzgS+E4m/ABMlzShXPEOx\ncM5Elq3dTmd3b6VDMTMbUZXsU5gFPFEwvSadtx9JF0haKmlpW1tb2QNbOGcSnd29rFjnU1PNrLpU\nMimoyLyi54FGxOURsSgiFrW2tpY5LFg4N+lsvsedzWZWZSqZFNYAcwqmZwNrKxTLPmZOaGDmhAb+\n+uhArV9mZmNPJZPC9cD56VlIxwPbImJdBePJk8TxR07hL6s2+SI2M6sq5Twl9WrgNuDpktZIequk\nCyVdmBb5ObAKWAl8E/i3csVyII4/agqb2jt5aP3OSodiZjZisuXacEQsHmR5ABeVa/8H64QjpwBw\n2yMbefph4yscjZnZyPAVzf2YM7mJuZObuOXhjZUOxcxsxDgpDOBlx07jTys30r6nu9KhmJmNCCeF\nAZyy4DA6u3u55aHyXxthZnYocFIYwPPnTWJiUy2/eWB9pUMxMxsRTgoDyNZkeOkx07jpbxvo6vEt\nL8xs7HNSGMQpCw5j2+4u7vCFbGZWBZwUBvH3T5tKfTbDr92EZGZVwElhEE11WV58dCu/XPYUPb2+\nutnMxjYnhRK89rmzeGp7B7c87LOQzGxsc1IowcuPnc6U5jqu+evqSodiZlZWTgolqMtmeN3zZvPb\nFRtYt213pcMxMysbJ4USnXf84UQES259rNKhmJmVjZNCieZMbuK0Z87g+7evZqdve2FmY5STwhC8\n7cVHsKOjm2uXPjF4YTOzUchJYQieM3cSiw6fxBV/ftSnp5rZmOSkMERve/ERPLF5Nzfed0iMHGpm\nNqycFIbolAWHccxh47n0Nw/5fkhmNuY4KQxRJiM+eOrTeXzTLn5wh/sWzGxscVI4ACc/fRqLDp/E\nZb99mG27uiodjpnZsClrUpB0qqQHJa2U9OEiy+dKulnS3ZLuk3R6OeMZLpL4+BnPYMuuTj5x4/JK\nh2NmNmzKlhQk1QBfAU4DFgCLJS3oU+zfgR9GxHOAs4Gvliue4fbMWRO46OT5XHfXkx6Ex8zGjHLW\nFF4ArIyIVRHRCVwDnNmnTAAt6esJwKg6pecdJ8/n2BktXHzd/WzcuafS4ZiZHbRyJoVZQGFP7Jp0\nXqGPA+dKWgP8HHhnsQ1JukDSUklL29oOnTuV1mUzXHrWcezo6OJfvnsnHV09lQ7JzOyglDMpqMi8\nvld8LQaWRMRs4HTgu5L2iykiLo+IRRGxqLW1tQyhHrhjZ7Tw+bOO487Ht3DxdfcT4YvazGz0ypZx\n22uAOQXTs9m/eeitwKkAEXGbpAZgKrChjHENu1c9eyar2tq59DcPMX/aOC46eX6lQzIzOyDlrCnc\nARwt6QhJdSQdydf3KbMaeBmApGOBBuDQaR8agne+dD5nLpzJZ3/1INffO6q6RszM8spWU4iIbknv\nAH4F1ABXRMRySZcASyPieuD9wDclvZekaenNMUrbXyTx6dc9m3VbO3jPNXeza083Z79gbqXDMjMb\nEo227+BFixbF0qVLKx1Gv3Z1dvOv37uLPzzUxodPO4YLX3JUpUMyM0PSnRGxaLByvqJ5mDXVZfnm\n+Yt49XEz+dQv/sZ//+8DvqOqmY0a5exorlp12QyXvXEhExtr+eYfH+XeNdu47I0LmTmxsdKhmZkN\nyDWFMqnJiEvOfAaff8NxLH9yG6d98Y/8ctm6SodlZjYgJ4UyksTrnjeb/33Xi5k3pYkLv3cXF193\nP7s6PZynmR2anBRGwLypzVx74Ylc+JKjuOaO1Zz8ud9zzV9X0+3xGMzsEOOkMELqshk+fNox/OjC\nE5g1sZEPX3c/p37xj/xy2VP0uiPazA4RTgoj7HmHT+bH/3oi3zjvefRGcOH37uTll/6Bq25/3PdO\nMrOK83UKFdTd08svlj3F5bes4v4ntzG5uY7zjj+cc144l2ktDZUOz8zGkFKvU3BSOAREBLc/uplv\n3rKKm/62gZqMOPnprZy1aA4nHzON2hpX6Mzs4JSaFHydwiFAEscfOYXjj5zCqrad/HDpGn581xp+\nu2IDk5vrOGXBdE571gxOPGqKE4SZlZVrCoeo7p5ebn6wjRvuXctNK9bT3tlDS0OWk4+ZxolHTeHE\no6YyZ3JTpcM0s1HCNYVRLluT4RULpvOKBdPp6OrhTw9v5OfL1nHLQxv52T3JXVhnT2rkhCOncOL8\nKZxw5FQOm+B+CDM7OK4pjDIRwcoNO7n1kU3c+shG/rJqM9t2dwFw5NRmjj9qCgvnTORZsyZw9LRx\nZN3cZGa4o7lq9PYGD6zbzm2PbOK2VZv466Ob2bknuWK6Ppvh2BktPGNmCwtmtrBgRgvHHNZCY11N\nhaM2s5HmpFClenuDVRvbWb52G/ev2cb9T27jgXXb2dGRJIqM4PApzRzVOo6jpiXP86eN46jWcUxo\nrK1w9GZWLu5TqFKZjJg/LfmiP3PhLCBpclqzZTcPrNvO8rXbWblhB49saOeWh9roLLjVxtRx9RzV\n2pxPErMnNTJzYvKY1FSLVGzYbTMbS5wUqoAk5kxuYs7kJl75jMPy87t7elmzZTePtO3kkbadrNyw\nk0fa2rnxvnX5foqchtoMMyY0MnNiQ/I8oYGZExuZMbGRWem85np/nMxGO/8vrmLZmgzzpjYzb2oz\nLzt2en5+RLC5vZO1Wzt4cutu1m3bzdqtu1m7rYN1W3fzp4c3smFHB31v2dTSkM3XLGakSWPmxAZm\nTmhk6vh6JjXVMaGxlpqMaxxmh6qyJgVJpwJfJBmj+VsR8akiZc4CPk4yRvO9EfFP5YzJBieJKePq\nmTKunmfNnlC0TFdPL+u3d7BuW0eSMLZ2pMkjmb579Ra27Orabz0JWhpqmdRUy8SmOiY11TKpqY6J\nTXVMbs7Nq9u7vDlZ3lDrznGzkVC2pCCpBvgK8ApgDXCHpOsj4oGCMkcDFwMviogtkqaVKx4bXrU1\nGWZPamL2pP4voNvd2cPabbtZt7WDjTv3sGVXJ1t2dbG14Llt5x4eWr+Trbs6ae/s/4aADbWZfPLY\nm0j2fZ7UJ6m0NNSSca3EbEjKWVN4AbAyIlYBSLoGOBN4oKDM24GvRMQWgIjYUMZ4bIQ11tUkZzm1\njiup/J7uHrbu6kqSR/ve5LFlV+c+iWTLri5WPLWdrel0f3cezwgmNO5NHOMbahnXkGV8fZbm+izj\nco+GfV8312UZn85rrs9Sl/W1HlY9ypkUZgFPFEyvAV7Yp8zTACT9maSJ6eMR8csyxmSHsPpsDdNb\napg+hDvE9vYGOzq62bKrk8255NGeSyT7Pm/Z1ckTm3exc083O/d0s2uAmkmhumwmnzSa63NJpYbm\n+iSBNNXX0FyXLGuur6GpLksPVb4pAAAJj0lEQVRTXQ2NdTU01tbQUJs8N9bW0FCXyc/zfazsUFTO\npFCs3t73N10WOBo4CZgN/FHSMyNi6z4bki4ALgCYO3fu8Edqo1YmIyY01TKhqZZ5NA9p3Z7eoL2z\nm50d3bTv6WbHnuR5Z0d3PnHs7OhmZ0GZnXu62dHRTdvOPTy+aRftnd3s2tNDe2d3vzWW/mQzShNF\nLnlkqM/WUJ/NUF/wui6bSebtM12Tlkle7y2Tob62Zu/rwmWF289mfIqxFVXOpLAGmFMwPRtYW6TM\nXyKiC3hU0oMkSeKOwkIRcTlwOSQXr5UtYqsqNRnR0pD0PRysiGBPdy/taQ2kvbObjq5ednf20NHV\nw+6uHnZ3Js8dXYXzevPzdnf20NnTy57uHvZ09bJ9dzed3el0d2/y6Epedw/DaH11NRlqa0RNRmRr\nMslzRn2e0/k1/czPTdeImkwmP53fbt9ymbTcftvbd/19l+87v9i+9ytXGGfNvvtxMhxYOZPCHcDR\nko4AngTOBvqeWfRTYDGwRNJUkuakVWWMyawsJNGQNgtNGYH99fRGPmF0dPXunzxyr7v2nZ8v19VL\nR3cPPT1Bd2/Q05t77t13uqf4/N1dPXvn77ON3iLbTOf3Bl09lf9NlxFkMxlQ8loICTJS0ryRe62k\nuUMSmXSBiqwD7F2/YJ2+6+fWUbqf5HXhfnPl9l0n/UdG4oyFM1n8gvK2lpQtKUREt6R3AL8i6S+4\nIiKWS7oEWBoR16fLTpH0ANAD/J+I2FSumMzGipqMkj6LUXgfq94iyWL/RLR3fndPP+XShFR0fm66\np/j8rp4gCAjojSCSl/nXUDg/eU4qZ7nX+65Dn/WDpPa47/rFliWve0tYJ/celJvvfWRmVgVKvfeR\nT38wM7M8JwUzM8tzUjAzszwnBTMzy3NSMDOzPCcFMzPLc1IwM7M8JwUzM8sbdRevSWoDHj/A1acC\nG4cxnEORj3H0G+vHB2P/GA/F4zs8IloHKzTqksLBkLS0lCv6RjMf4+g31o8Pxv4xjubjc/ORmZnl\nOSmYmVletSWFyysdwAjwMY5+Y/34YOwf46g9vqrqUzAzs4FVW03BzMwG4KRgZmZ5VZMUJJ0q6UFJ\nKyV9uNLxHChJV0jaIGlZwbzJkn4j6eH0eVI6X5K+lB7zfZKeW7nISyNpjqSbJa2QtFzSu9P5Y+IY\nJTVI+quke9Pj+0Q6/whJt6fH9wNJden8+nR6Zbp8XiXjHwpJNZLulnRjOj2mjlHSY5Lul3SPpKXp\nvFH/Oa2KpCCpBvgKcBqwAFgsaUFlozpgS4BT+8z7MHBTRBwN3JROQ3K8R6ePC4CvjVCMB6MbeH9E\nHAscD1yU/q3GyjHuAV4aEccBC4FTJR0PfBr4Qnp8W4C3puXfCmyJiPnAF9Jyo8W7gRUF02PxGE+O\niIUF1ySM/s9pMibo2H4AJwC/Kpi+GLi40nEdxPHMA5YVTD8IzEhfzwAeTF9/A1hcrNxoeQA/A14x\nFo8RaALuAl5IcvVrNp2f/7ySjGN+Qvo6m5ZTpWMv4dhmk3wpvhS4kWTs+bF2jI8BU/vMG/Wf06qo\nKQCzgCcKptek88aK6RGxDiB9npbOH9XHnTYjPAe4nTF0jGmzyj3ABuA3wCPA1ojoTosUHkP++NLl\n24ApIxvxAbkM+CDQm05PYewdYwC/lnSnpAvSeaP+c5qtdAAjREXmVcO5uKP2uCWNA34MvCcitkvF\nDiUpWmTeIX2MEdEDLJQ0EfgJcGyxYunzqDs+Sa8CNkTEnZJOys0uUnTUHmPqRRGxVtI04DeS/jZA\n2VFzjNVSU1gDzCmYng2srVAs5bBe0gyA9HlDOn9UHrekWpKEcFVEXJfOHlPHCBARW4Hfk/SdTJSU\n+5FWeAz540uXTwA2j2ykQ/Yi4AxJjwHXkDQhXcbYOkYiYm36vIEkub+AMfA5rZakcAdwdHr2Qx1w\nNnB9hWMaTtcDb0pfv4mkHT43//z0zIfjgW25qu2hSkmV4NvAioi4tGDRmDhGSa1pDQFJjcDLSTpj\nbwZenxbre3y543498LtIG6UPVRFxcUTMjoh5JP/XfhcR5zCGjlFSs6TxudfAKcAyxsLntNKdGiP1\nAE4HHiJpv/2/lY7nII7jamAd0EXy6+OtJO2vNwEPp8+T07IiOevqEeB+YFGl4y/h+P6OpFp9H3BP\n+jh9rBwj8Gzg7vT4lgEfTecfCfwVWAlcC9Sn8xvS6ZXp8iMrfQxDPN6TgBvH2jGmx3Jv+lie+04Z\nC59T3+bCzMzyqqX5yMzMSuCkYGZmeU4KZmaW56RgZmZ5TgpmZpbnpGBmZnlOCmYlkLRQ0ukF02do\nmG7BLuk9kpqGY1tmB8vXKZiVQNKbSS44ekcZtv1Yuu2NQ1inJpJ7KJkNK9cUbEyRNC8doOeb6SA2\nv05vJ1Gs7FGSfpne5fKPko5J579B0rJ0IJxb0lujXAK8MR1Q5Y2S3izpy2n5JZK+pmRwoFWSXqJk\nMKQVkpYU7O9rkpZq38F13gXMBG6WdHM6b3E6eMsySZ8uWH+npEsk3Q6cIOlTkh5IB235XHneUas6\nlb6k2g8/hvNBMtZEN7Awnf4hcG4/ZW8Cjk5fv5DknjuQ3IZgVvp6Yvr8ZuDLBevmp0kGPrqG5FYG\nZwLbgWeR/Oi6syCW3C0PakhuhPfsdPox0vvykySI1UAryV2Mfwe8Jl0WwFm5bZHck1+Fcfrhx8E+\nXFOwsejRiLgnfX0nSaLYR3pr7hOBa9OxDb5BMigKwJ+BJZLeTvIFXoobIiJIEsr6iLg/InpJ7ouT\n2/9Zku4iuffRM0hGAezr+cDvI6ItkrEFrgL+Pl3WQ3L3WEgSTwfwLUmvBXaVGKfZgKplPAWrLnsK\nXvcAxZqPMiSDvizsuyAiLpT0QuAfgHsk7VdmgH329tl/L5CVdATwAeD5EbElbVZqKLKdfgeOADoi\n7UeIiG5JLwBeRnIn0neQ3KLa7KC4pmBVKSK2A49KegPkB1Y/Ln19VETcHhEfJRkacg6wAxh/ELts\nAdqBbZKmk4zZm1O47duBl0iamo4tvhj4Q9+NpTWdCRHxc+A9JOM9mx001xSsmp0DfE3SvwO1JP0C\n9wKflXQ0ya/2m9J5q4EPp01NnxzqjiLiXkl3kzQnrSJposq5HPiFpHURcbKki0nGHhDw84j42f5b\nZDzwM0kNabn3DjUms2J8SqqZmeW5+cjMzPLcfGRjnqSvkIwbXOiLEXFlJeIxO5S5+cjMzPLcfGRm\nZnlOCmZmluekYGZmeU4KZmaW9/8BzZIkv3Sf+TUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10c468c90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#cv_result = pd.DataFrame.from_csv('lgbm1_nestimators.csv')\n",
    "cv_result = pd.read_json(\"lgbm_1.json\")\n",
    "\n",
    "# plot\n",
    "test_means = cv_result['multi_logloss-mean']\n",
    "#test_stds = cv_result['multi_logloss-std'] \n",
    "\n",
    "x_axis = range(0, cv_result.shape[0])\n",
    "pyplot.plot(x_axis, test_means) \n",
    "\n",
    "pyplot.title(\"LightGBM n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'lgbm1_n_estimators.png')\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
